#### Setting environment ####
require(irr)

#### Data preparation ####
# Original voting records
court.opinion.data <- read.csv("court_opinion_data.csv")
# Coder #1's data
coding.data.1 <- read.csv("coding_data_1.csv")
# Coder #2's data
coding.data.2 <- read.csv("coding_data_2.csv")
# Coder #3's data
coding.data.3 <- read.csv("coding_data_3.csv")
# finalized data
voting.records <- read.csv("voting_records.csv")

#### Inter-rater reliability ####
table(table(coding.data.1$filename))
table(table(coding.data.2$filename))
table(table(coding.data.3$filename))
# Maximum number of issues per case in the coded data is three.

## Arrange data to calcurate inter-rater reliability
# "irr.data.1.2," "irr.data.1.3," and "irr.data.2.3" stack the justice-level codings vertically.
# 1 indicates that the justice takes the minority side, and 0 indicates otherwise.
# Values from 2 to 7 are bogus values and never consistent across coders.
irr.data.1.2 <- irr.data.1.3 <- irr.data.2.3 <- c()
# "consistency" records consistency at the case level.
# The first, second, and third columns correspond to the #1-#2, #1-#3, and #2-#3 comparison.
consistency <- matrix(NA, nrow(court.opinion.data), 3)
for (i in 1:nrow(court.opinion.data)) {  # iterate by case
  # Column id of justices who wrote a concurring or dissenting opinion in the case
  coded.justices <- which(court.opinion.data[i, ] == 0)
  # Extract coded data of the case
  coder.1 <- as.matrix(subset(coding.data.1, filename == court.opinion.data$filename[i])[, coded.justices])
  coder.2 <- as.matrix(subset(coding.data.2, filename == court.opinion.data$filename[i])[, coded.justices])
  coder.3 <- as.matrix(subset(coding.data.3, filename == court.opinion.data$filename[i])[, coded.justices])
  # If data is missing, fill bogus values.
  if (nrow(coder.1) == 0) {
    coder.1 <- matrix(5, 1, length(coded.justices))
  }
  if (nrow(coder.2) == 0) {
    coder.2 <- matrix(6, 1, length(coded.justices))
  }
  if (nrow(coder.3) == 0) {
    coder.3 <- matrix(7, 1, length(coded.justices))
  }
  # If coders mistakenly coded the case as unanimous, fill bogus values.
  if (sum(coder.1 > 0) == 0) {
    coder.1[coder.1 == 0] <- 5
  }
  if (sum(coder.2 > 0) == 0) {
    coder.2[coder.2 == 0] <- 6
  }
  if (sum(coder.3 > 0) == 0) {
    coder.3[coder.3 == 0] <- 7
  }
  names(coder.1) <- names(coder.2) <- names(coder.3) <- NULL
  # Number of issues identified by each coder
  n.rows.1 <- nrow(coder.1)
  n.rows.2 <- nrow(coder.2)
  n.rows.3 <- nrow(coder.3)
  # Comparison between Coders #1 and #2
  if (n.rows.1 == 1 & n.rows.2 == 1) {  # if the number of issues identified by both coders is the same
    coder.1.r <- as.vector(coder.1)
    coder.2.r <- as.vector(coder.2)
    irr.data.1.2 <- rbind(irr.data.1.2, cbind(coder.1.r, coder.2.r))
    consistency[i, 1] <- sum(coder.1.r != coder.2.r) == 0
  } else {  # if the number of issues identified by each coder differs
    # Fill bogus values for issues not identified by one of the coders
    if (n.rows.1 < n.rows.2) {
      coder.1.r <- rbind(coder.1, matrix(2, n.rows.2 - n.rows.1, ncol(coder.1)))
      coder.2.r <- coder.2
    } else if (n.rows.2 < n.rows.1) {
      coder.1.r <- coder.1
      coder.2.r <- rbind(coder.2, matrix(3, n.rows.1 - n.rows.2, ncol(coder.2)))
    } else {
      coder.1.r <- coder.1
      coder.2.r <- coder.2
    }
    # If at least one of the two coders considered that the case included
    # multiple issues, I counted the number of consistent codings for
    # every possible combination of issues ("cand.X"s represent such combinations)
    # and adopted the combination recording the greater consistency.
    if (max(n.rows.1, n.rows.2) == 2) {
      cand.1 <- sum(coder.1.r[1, ] != coder.2.r[1, ]) + sum(coder.1.r[2, ] != coder.2.r[2, ])
      cand.2 <- sum(coder.1.r[1, ] != coder.2.r[2, ]) + sum(coder.1.r[2, ] != coder.2.r[1, ])
      if (min(cand.1, cand.2) == cand.1) {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ]), 
                                    c(coder.2.r[1, ], coder.2.r[2, ])))
        consistency[i, 1] <- cand.1 == 0
      } else {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ]), 
                                    c(coder.2.r[2, ], coder.2.r[2, ])))
        consistency[i, 1] <- cand.2 == 0
      }
    } else {  # number of rows is three
      cand.1 <- sum(coder.1.r[1, ] != coder.2.r[1, ]) + 
        sum(coder.1.r[2, ] != coder.2.r[2, ]) + sum(coder.1.r[3, ] != coder.2.r[3, ])
      cand.2 <- sum(coder.1.r[1, ] != coder.2.r[1, ]) + 
        sum(coder.1.r[2, ] != coder.2.r[3, ]) + sum(coder.1.r[3, ] != coder.2.r[2, ])
      cand.3 <- sum(coder.1.r[1, ] != coder.2.r[2, ]) + 
        sum(coder.1.r[2, ] != coder.2.r[1, ]) + sum(coder.1.r[3, ] != coder.2.r[3, ])
      cand.4 <- sum(coder.1.r[1, ] != coder.2.r[2, ]) + 
        sum(coder.1.r[2, ] != coder.2.r[3, ]) + sum(coder.1.r[3, ] != coder.2.r[1, ])
      cand.5 <- sum(coder.1.r[1, ] != coder.2.r[3, ]) + 
        sum(coder.1.r[2, ] != coder.2.r[1, ]) + sum(coder.1.r[3, ] != coder.2.r[2, ])
      cand.6 <- sum(coder.1.r[1, ] != coder.2.r[3, ]) + 
        sum(coder.1.r[2, ] != coder.2.r[2, ]) + sum(coder.1.r[3, ] != coder.2.r[1, ])
      if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.1) {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.2.r[1, ], coder.2.r[2, ], coder.2.r[3, ])))
        consistency[i, 1] <- cand.1 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.2) {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.2.r[1, ], coder.2.r[3, ], coder.2.r[2, ])))
        consistency[i, 1] <- cand.2 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.3) {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.2.r[2, ], coder.2.r[1, ], coder.2.r[3, ])))
        consistency[i, 1] <- cand.3 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.4) {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.2.r[2, ], coder.2.r[3, ], coder.2.r[1, ])))
        consistency[i, 1] <- cand.4 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.5) {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.2.r[3, ], coder.2.r[1, ], coder.2.r[2, ])))
        consistency[i, 1] <- cand.5 == 0
      } else {
        irr.data.1.2 <- rbind(irr.data.1.2, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.2.r[3, ], coder.2.r[2, ], coder.2.r[1, ])))
        consistency[i, 1] <- cand.6 == 0
      }
    }
  }
  # Comparison between Coders #1 and #3
  if (n.rows.1 == 1 & n.rows.3 == 1) {  # # if the number of issues identified by both coders is the same
    coder.1.r <- as.vector(coder.1)
    coder.3.r <- as.vector(coder.3)
    irr.data.1.3 <- rbind(irr.data.1.3, cbind(coder.1.r, coder.3.r))
    consistency[i, 2] <- sum(coder.1.r != coder.3.r) == 0
  } else {  # if the number of issues identified by each coder differs
    # Fill bogus values for issues not identified by one of the coders
    if (n.rows.1 < n.rows.3) {
      coder.1.r <- rbind(coder.1, matrix(2, n.rows.3 - n.rows.1, ncol(coder.1)))
      coder.3.r <- coder.3
    } else if (n.rows.3 < n.rows.1) {
      coder.1.r <- coder.1
      coder.3.r <- rbind(coder.3, matrix(4, n.rows.1 - n.rows.3, ncol(coder.3)))
    } else {
      coder.1.r <- coder.1
      coder.3.r <- coder.3
    }
    # If at least one of the two coders considered that the case included
    # multiple issues, I counted the number of consistent codings for
    # every possible combination of issues ("cand.X"s represent such combinations)
    # and adopted the combination recording the greater consistency.
    if (max(n.rows.1, n.rows.3) == 2) {
      cand.1 <- sum(coder.1.r[1, ] != coder.3.r[1, ]) + sum(coder.1.r[2, ] != coder.3.r[2, ])
      cand.2 <- sum(coder.1.r[1, ] != coder.3.r[2, ]) + sum(coder.1.r[2, ] != coder.3.r[1, ])
      if (min(cand.1, cand.2) == cand.1) {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ]), 
                                    c(coder.3.r[1, ], coder.3.r[2, ])))
        consistency[i, 2] <- cand.1 == 0
      } else {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ]), 
                                    c(coder.3.r[2, ], coder.3.r[2, ])))
        consistency[i, 2] <- cand.2 == 0
      }
    } else {  # number of rows is three
      cand.1 <- sum(coder.1.r[1, ] != coder.3.r[1, ]) + 
        sum(coder.1.r[2, ] != coder.3.r[2, ]) + sum(coder.1.r[3, ] != coder.3.r[3, ])
      cand.2 <- sum(coder.1.r[1, ] != coder.3.r[1, ]) + 
        sum(coder.1.r[2, ] != coder.3.r[3, ]) + sum(coder.1.r[3, ] != coder.3.r[2, ])
      cand.3 <- sum(coder.1.r[1, ] != coder.3.r[2, ]) + 
        sum(coder.1.r[2, ] != coder.3.r[1, ]) + sum(coder.1.r[3, ] != coder.3.r[3, ])
      cand.4 <- sum(coder.1.r[1, ] != coder.3.r[2, ]) + 
        sum(coder.1.r[2, ] != coder.3.r[3, ]) + sum(coder.1.r[3, ] != coder.3.r[1, ])
      cand.5 <- sum(coder.1.r[1, ] != coder.3.r[3, ]) + 
        sum(coder.1.r[2, ] != coder.3.r[1, ]) + sum(coder.1.r[3, ] != coder.3.r[2, ])
      cand.6 <- sum(coder.1.r[1, ] != coder.3.r[3, ]) + 
        sum(coder.1.r[2, ] != coder.3.r[2, ]) + sum(coder.1.r[3, ] != coder.3.r[1, ])
      if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.1) {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.3.r[1, ], coder.3.r[2, ], coder.3.r[3, ])))
        consistency[i, 2] <- cand.1 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.2) {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.3.r[1, ], coder.3.r[3, ], coder.3.r[2, ])))
        consistency[i, 2] <- cand.2 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.3) {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.3.r[2, ], coder.3.r[1, ], coder.3.r[3, ])))
        consistency[i, 2] <- cand.3 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.4) {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.3.r[2, ], coder.3.r[3, ], coder.3.r[1, ])))
        consistency[i, 2] <- cand.4 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.5) {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.3.r[3, ], coder.3.r[1, ], coder.3.r[2, ])))
        consistency[i, 2] <- cand.5 == 0
      } else {
        irr.data.1.3 <- rbind(irr.data.1.3, 
                              cbind(c(coder.1.r[1, ], coder.1.r[2, ], coder.1.r[3, ]), 
                                    c(coder.3.r[3, ], coder.3.r[2, ], coder.3.r[1, ])))
        consistency[i, 2] <- cand.6 == 0
      }
    }
  }
  # Comparison between Coders #2 and #3
  if (n.rows.2 == 1 & n.rows.3 == 1) {  # # if the number of issues identified by both coders is the same
    coder.2.r <- as.vector(coder.2)
    coder.3.r <- as.vector(coder.3)
    irr.data.2.3 <- rbind(irr.data.2.3, cbind(coder.2.r, coder.3.r))
    consistency[i, 3] <- sum(coder.2.r != coder.3.r) == 0
  } else {  # if the number of issues identified by each coder differs
    # Fill bogus values for issues not identified by one of the coders
    if (n.rows.2 < n.rows.3) {
      coder.2.r <- rbind(coder.2, matrix(3, n.rows.3 - n.rows.2, ncol(coder.2)))
      coder.3.r <- coder.3
    } else if (n.rows.3 < n.rows.2) {
      coder.2.r <- coder.2
      coder.3.r <- rbind(coder.3, matrix(4, n.rows.2 - n.rows.3, ncol(coder.3)))
    } else {
      coder.2.r <- coder.2
      coder.3.r <- coder.3
    }
    # If at least one of the two coders considered that the case included
    # multiple issues, I counted the number of consistent codings for
    # every possible combination of issues ("cand.X"s represent such combinations)
    # and adopted the combination recording the greater consistency.
    if (max(n.rows.2, n.rows.3) == 2) {
      cand.1 <- sum(coder.2.r[1, ] != coder.3.r[1, ]) + sum(coder.2.r[2, ] != coder.3.r[2, ])
      cand.2 <- sum(coder.2.r[1, ] != coder.3.r[2, ]) + sum(coder.2.r[2, ] != coder.3.r[1, ])
      if (min(cand.1, cand.2) == cand.1) {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ]), 
                                    c(coder.3.r[1, ], coder.3.r[2, ])))
        consistency[i, 3] <- cand.1 == 0
      } else {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ]), 
                                    c(coder.3.r[2, ], coder.3.r[2, ])))
        consistency[i, 3] <- cand.2 == 0
      }
    } else {  # number of rows is three
      cand.1 <- sum(coder.2.r[1, ] != coder.3.r[1, ]) + 
        sum(coder.2.r[2, ] != coder.3.r[2, ]) + sum(coder.2.r[3, ] != coder.3.r[3, ])
      cand.2 <- sum(coder.2.r[1, ] != coder.3.r[1, ]) + 
        sum(coder.2.r[2, ] != coder.3.r[3, ]) + sum(coder.2.r[3, ] != coder.3.r[2, ])
      cand.3 <- sum(coder.2.r[1, ] != coder.3.r[2, ]) + 
        sum(coder.2.r[2, ] != coder.3.r[1, ]) + sum(coder.2.r[3, ] != coder.3.r[3, ])
      cand.4 <- sum(coder.2.r[1, ] != coder.3.r[2, ]) + 
        sum(coder.2.r[2, ] != coder.3.r[3, ]) + sum(coder.2.r[3, ] != coder.3.r[1, ])
      cand.5 <- sum(coder.2.r[1, ] != coder.3.r[3, ]) + 
        sum(coder.2.r[2, ] != coder.3.r[1, ]) + sum(coder.2.r[3, ] != coder.3.r[2, ])
      cand.6 <- sum(coder.2.r[1, ] != coder.3.r[3, ]) + 
        sum(coder.2.r[2, ] != coder.3.r[2, ]) + sum(coder.2.r[3, ] != coder.3.r[1, ])
      if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.1) {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ], coder.2.r[3, ]), 
                                    c(coder.3.r[1, ], coder.3.r[2, ], coder.3.r[3, ])))
        consistency[i, 3] <- cand.1 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.2) {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ], coder.2.r[3, ]), 
                                    c(coder.3.r[1, ], coder.3.r[3, ], coder.3.r[2, ])))
        consistency[i, 3] <- cand.2 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.3) {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ], coder.2.r[3, ]), 
                                    c(coder.3.r[2, ], coder.3.r[1, ], coder.3.r[3, ])))
        consistency[i, 3] <- cand.3 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.4) {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ], coder.2.r[3, ]), 
                                    c(coder.3.r[2, ], coder.3.r[3, ], coder.3.r[1, ])))
        consistency[i, 3] <- cand.4 == 0
      } else if (min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == cand.5) {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ], coder.2.r[3, ]), 
                                    c(coder.3.r[3, ], coder.3.r[1, ], coder.3.r[2, ])))
        consistency[i, 3] <- cand.5 == 0
      } else {
        irr.data.2.3 <- rbind(irr.data.2.3, 
                              cbind(c(coder.2.r[1, ], coder.2.r[2, ], coder.2.r[3, ]), 
                                    c(coder.3.r[3, ], coder.3.r[2, ], coder.3.r[1, ])))
        consistency[i, 3] <- cand.6 == 0
      }
    }
  }
}

# Compute inter-rater reliability
kappa2(irr.data.1.2)
kappa2(irr.data.1.3)
kappa2(irr.data.2.3)

# Compute inter-rater reliability excluding missing codes
kappa2(irr.data.1.2[rowSums(irr.data.1.2 > 4) == 0, ])
kappa2(irr.data.1.3[rowSums(irr.data.1.3 > 4) == 0, ])
kappa2(irr.data.2.3[rowSums(irr.data.2.3 > 4) == 0, ])

#### Description of coding discrepancies ####
# Total number of cases considered for analysis
nrow(court.opinion.data)

## Cases of unanimity across the coders
sum(rowSums(consistency) == 3)
round(mean(rowSums(consistency) == 3), 3)

## Cases with plurality decision
sum(rowSums(consistency) == 1)
round(mean(rowSums(consistency) == 1), 3)
# Cases where Coder #1 disagreed with the other two coders
sum(consistency[, 1] == 0 & consistency[, 2] == 0 & consistency[, 3] == 1)
round(mean(consistency[, 1] == 0 & consistency[, 2] == 0 & consistency[, 3] == 1), 3)
# Cases where Coder #2 disagreed with the other two coders
sum(consistency[, 1] == 0 & consistency[, 2] == 1 & consistency[, 3] == 0)
round(mean(consistency[, 1] == 0 & consistency[, 2] == 1 & consistency[, 3] == 0), 3)
# Cases where Coder #3 disagreed with the other two coders
sum(consistency[, 1] == 1 & consistency[, 2] == 0 & consistency[, 3] == 0)
round(mean(consistency[, 1] == 1 & consistency[, 2] == 0 & consistency[, 3] == 0), 3)

## Cases with no consistency across any coders
sum(rowSums(consistency) == 0)
round(mean(rowSums(consistency) == 0), 3)

#### Identify whose coding was adopted for cases with complete inconsistency ####
# List the ids of cases with complete inconsistency
inconsistent.cases <- court.opinion.data$filename[which(rowSums(consistency) == 0)]

consistency.with.voting.records <- matrix(NA, sum(rowSums(consistency) == 0), 3)
rownames(consistency.with.voting.records) <- inconsistent.cases
for (i in 1:length(inconsistent.cases)) {  # iterate by case
  # Column ids of justices who wrote a concurring or dissenting opinion in the case
  coded.justices <- which(court.opinion.data[court.opinion.data$filename == inconsistent.cases[i], ] == 0)
  # Extract coded data of the case
  coder.1 <- as.matrix(subset(coding.data.1, filename == inconsistent.cases[i])[, coded.justices])
  coder.2 <- as.matrix(subset(coding.data.2, filename == inconsistent.cases[i])[, coded.justices])
  coder.3 <- as.matrix(subset(coding.data.3, filename == inconsistent.cases[i])[, coded.justices])
  # One is added to the column ids because "voting.records" has an additional column
  finalized.coding <- as.matrix(subset(voting.records, filename == inconsistent.cases[i])[, coded.justices + 1])
  names(coder.1) <- names(coder.2) <- names(coder.3) <- names(finalized.coding) <- NULL
  # Number of issues identified by each coder
  n.rows.1 <- nrow(coder.1)
  n.rows.2 <- nrow(coder.2)
  n.rows.3 <- nrow(coder.3)
  n.rows.final <- nrow(finalized.coding)
  if (n.rows.1 == 0 | sum(coder.1 > 0) == 0) {
    # Record NA if the coder missed coding any issues
    consistency.with.voting.records[i, 1] <- NA
  } else if (n.rows.1 != n.rows.final) {
    # Record FALSE if the number of issues differs from the finalized coding
    consistency.with.voting.records[i, 1] <- FALSE
  } else if (n.rows.1 == 1 & n.rows.final == 1) {
    # Compare coder's codes with finalized codes when only one issue is identified
    consistency.with.voting.records[i, 1] <- sum(coder.1 != finalized.coding) == 0
  } else if (n.rows.1 == 2 & n.rows.final == 2) {
    # Consider all possible combinations of issue codings when more than one issue is identified
    cand.1 <- sum(coder.1[1, ] != finalized.coding[1, ]) + sum(coder.1[2, ] != finalized.coding[2, ])
    cand.2 <- sum(coder.1[1, ] != finalized.coding[2, ]) + sum(coder.1[2, ] != finalized.coding[1, ])
    consistency.with.voting.records[i, 1] <- min(cand.1, cand.2) == 0
  } else {  # number of rows is three
    cand.1 <- sum(coder.1[1, ] != finalized.coding[1, ]) + 
      sum(coder.1[2, ] != finalized.coding[2, ]) + sum(coder.1[3, ] != finalized.coding[3, ])
    cand.2 <- sum(coder.1[1, ] != finalized.coding[1, ]) + 
      sum(coder.1[2, ] != finalized.coding[3, ]) + sum(coder.1[3, ] != finalized.coding[2, ])
    cand.3 <- sum(coder.1[1, ] != finalized.coding[2, ]) + 
      sum(coder.1[2, ] != finalized.coding[1, ]) + sum(coder.1[3, ] != finalized.coding[3, ])
    cand.4 <- sum(coder.1[1, ] != finalized.coding[2, ]) + 
      sum(coder.1[2, ] != finalized.coding[3, ]) + sum(coder.1[3, ] != finalized.coding[1, ])
    cand.5 <- sum(coder.1[1, ] != finalized.coding[3, ]) + 
      sum(coder.1[2, ] != finalized.coding[1, ]) + sum(coder.1[3, ] != finalized.coding[2, ])
    cand.6 <- sum(coder.1[1, ] != finalized.coding[3, ]) + 
      sum(coder.1[2, ] != finalized.coding[2, ]) + sum(coder.1[3, ] != finalized.coding[1, ])
    consistency.with.voting.records[i, 1] <- min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == 0
  }
  if (n.rows.2 == 0 | sum(coder.2 > 0) == 0) {
    # Record NA if the coder missed coding any issues
    consistency.with.voting.records[i, 2] <- NA
  } else if (n.rows.2 != n.rows.final) {
    # Record FALSE if the number of issues differs from the finalized coding
    consistency.with.voting.records[i, 2] <- FALSE
  } else if (n.rows.2 == 1 & n.rows.final == 1) {
    # Compare coder's codes with finalized codes when only one issue is identified
    consistency.with.voting.records[i, 2] <- sum(coder.2 != finalized.coding) == 0
  } else if (n.rows.2 == 2 & n.rows.final == 2) {
    # Consider all possible combinations of issue codings when more than one issue is identified
    cand.1 <- sum(coder.2[1, ] != finalized.coding[1, ]) + sum(coder.2[2, ] != finalized.coding[2, ])
    cand.2 <- sum(coder.2[1, ] != finalized.coding[2, ]) + sum(coder.2[2, ] != finalized.coding[1, ])
    consistency.with.voting.records[i, 2] <- min(cand.1, cand.2) == 0
  } else {  # number of rows is three
    cand.1 <- sum(coder.2[1, ] != finalized.coding[1, ]) + 
      sum(coder.2[2, ] != finalized.coding[2, ]) + sum(coder.2[3, ] != finalized.coding[3, ])
    cand.2 <- sum(coder.2[1, ] != finalized.coding[1, ]) + 
      sum(coder.2[2, ] != finalized.coding[3, ]) + sum(coder.2[3, ] != finalized.coding[2, ])
    cand.3 <- sum(coder.2[1, ] != finalized.coding[2, ]) + 
      sum(coder.2[2, ] != finalized.coding[1, ]) + sum(coder.2[3, ] != finalized.coding[3, ])
    cand.4 <- sum(coder.2[1, ] != finalized.coding[2, ]) + 
      sum(coder.2[2, ] != finalized.coding[3, ]) + sum(coder.2[3, ] != finalized.coding[1, ])
    cand.5 <- sum(coder.2[1, ] != finalized.coding[3, ]) + 
      sum(coder.2[2, ] != finalized.coding[1, ]) + sum(coder.2[3, ] != finalized.coding[2, ])
    cand.6 <- sum(coder.2[1, ] != finalized.coding[3, ]) + 
      sum(coder.2[2, ] != finalized.coding[2, ]) + sum(coder.2[3, ] != finalized.coding[1, ])
    consistency.with.voting.records[i, 2] <- min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == 0
  }
  if (n.rows.3 == 0 | sum(coder.3 > 0) == 0) {
    # Record NA if the coder missed coding any issues
    consistency.with.voting.records[i, 3] <- NA
  } else if (n.rows.3 != n.rows.final) {
    # Record FALSE if the number of issues differs from the finalized coding
    consistency.with.voting.records[i, 3] <- FALSE
  } else if (n.rows.3 == 1 & n.rows.final == 1) {
    # Compare coder's codes with finalized codes when only one issue is identified
    consistency.with.voting.records[i, 3] <- sum(coder.3 != finalized.coding) == 0
  } else if (n.rows.3 == 2 & n.rows.final == 2) {
    # Consider all possible combinations of issue codings when more than one issue is identified
    cand.1 <- sum(coder.3[1, ] != finalized.coding[1, ]) + sum(coder.3[2, ] != finalized.coding[2, ])
    cand.2 <- sum(coder.3[1, ] != finalized.coding[2, ]) + sum(coder.3[2, ] != finalized.coding[1, ])
    consistency.with.voting.records[i, 3] <- min(cand.1, cand.2) == 0
  } else {  # number of rows is three
    cand.1 <- sum(coder.3[1, ] != finalized.coding[1, ]) + 
      sum(coder.3[2, ] != finalized.coding[2, ]) + sum(coder.3[3, ] != finalized.coding[3, ])
    cand.2 <- sum(coder.3[1, ] != finalized.coding[1, ]) + 
      sum(coder.3[2, ] != finalized.coding[3, ]) + sum(coder.3[3, ] != finalized.coding[2, ])
    cand.3 <- sum(coder.3[1, ] != finalized.coding[2, ]) + 
      sum(coder.3[2, ] != finalized.coding[1, ]) + sum(coder.3[3, ] != finalized.coding[3, ])
    cand.4 <- sum(coder.3[1, ] != finalized.coding[2, ]) + 
      sum(coder.3[2, ] != finalized.coding[3, ]) + sum(coder.3[3, ] != finalized.coding[1, ])
    cand.5 <- sum(coder.3[1, ] != finalized.coding[3, ]) + 
      sum(coder.3[2, ] != finalized.coding[1, ]) + sum(coder.3[3, ] != finalized.coding[2, ])
    cand.6 <- sum(coder.3[1, ] != finalized.coding[3, ]) + 
      sum(coder.3[2, ] != finalized.coding[2, ]) + sum(coder.3[3, ] != finalized.coding[1, ])
    consistency.with.voting.records[i, 3] <- min(cand.1, cand.2, cand.3, cand.4, cand.5, cand.6) == 0
  }
}

# "consistency.with.voting.records" takes TRUE if the coder's view
# (corresponding to columns) was adopted in the finalized coding.
consistency.with.voting.records

# Flag cases coded solely by the author
flagged.cases <- inconsistent.cases[which(consistency.with.voting.records[, 3] | 
                                            rowSums(consistency.with.voting.records) == 0)]
voting.records$author <- (voting.records$filename %in% flagged.cases) * 1

# write.csv(voting.records, file = "voting_records.csv", row.names = FALSE)